#!/usr/bin/env python3
# -*- coding: utf-8 -*-
import requests
import pandas as pd
import os
import io
import datetime
from requests.adapters import HTTPAdapter
import time
import re

def get_list(date):
    return datetime.datetime.strptime(date, "%Y-%m-%d").timestamp()

if __name__=='__main__':
    ark_holdings_data_address='arkholding/'

    arkholding_files=os.listdir('arkholding')
    arkholding_files_date=[file[12:22] for file in arkholding_files]
    arkholding_files_date_sort=sorted(arkholding_files_date, key=lambda date: get_list(date))
    new_arkholding_file_date = arkholding_files_date_sort[-1]
    new_arkholding_file = 'Ark_holding_' + new_arkholding_file_date + '.txt'

    new_data=open(ark_holdings_data_address+new_arkholding_file,'r',encoding='utf-8').read()
    new_data_fund_res=re.findall('ARKF|ARKG|ARKK|ARKQ|ARKW|ARKX|PRNT|IZRL',new_data,re.IGNORECASE)
    new_data_fund_res=list(set(new_data_fund_res))
    new_data_fund_res=[i.upper() for i in new_data_fund_res]
    new_data_fund_res = list(set(new_data_fund_res))

    today = datetime.datetime.today()
    yesterday = today - datetime.timedelta(days=1)
    yesterday=str(yesterday.date())

    ark_type=['ARKF','ARKG','ARKK','ARKQ','ARKW','ARKX','PRNT','IZRL','test']

    diff = set(ark_type).difference(set(new_data_fund_res))
    print(type(diff),diff)
    crawl_ark_type=list(diff)
    print(crawl_ark_type,type(crawl_ark_type))
    if yesterday!=new_arkholding_file_date or len(new_data_fund_res)<8:
        print('not same')
        print(yesterday,type(yesterday))
        print('data_date=',new_arkholding_file_date)